package com.chance.cc.crawler.development.scripts.kuaihe;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpConstant;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.util.*;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.core.downloader.HttpRequestBody.ContentType.FORM;

/**
 * @Author Zhao.Hhuan
 * @Date Create in 2021/1/22 17:33
 * @Description 1919 采集商品信息
 **/
public class KuaiheCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(KuaiheCrawlerScript.class);

    private static String categoriesListUrl = "https://kuaihe-720.1919.cn/sys/init.do";
    private static String goodsListUrl = "https://kuaihe-720.1919.cn/product/categoryProductId.do\\S*";
    private static String goodsDetailUrl = "https://kuaihe-720.1919.cn/product/detail.do\\S*";


    private static final String requestAgainTag = "kuaihe_download_retry_count";


    /**
     * 脚本domain定义
     *
     * @return
     */
    @Override
    public String domain() {
        return "kuaihe";
    }

    /**
     * 进入脚本的正则列表
     */
    @Override
    public void initUrlRegulars() {
        addUrlRegular(categoriesListUrl);
        addUrlRegular(goodsListUrl);
        addUrlRegular(goodsDetailUrl);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     *
     * @param crawlerRequestRecord
     * @return
     */
    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }

    /**
     * 解析链接方法
     *
     * @param crawlerRequestRecord
     * @param httpPage
     * @return
     */
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();

        if (!httpPage.isDownloadSuccess()) {
            log.error("kuaihe down load page is error! will retry");
            requestAgainCrawlerRecord(parsedLinks, crawlerRequestRecord);
            return parsedLinks;
        }

        String requestUrl = httpPage.getRequest().getUrl();
        if(requestUrl.matches(categoriesListUrl)){
            List<String> categories = new ArrayList<>();
            try {
                categories = httpPage.getJson().jsonPath($_type + ".data.categories").all();
            } catch (Exception e) {
                log.error("categories page down load is fail ! will retry");
                requestAgainCrawlerRecord(parsedLinks,crawlerRequestRecord);
                return parsedLinks;
            }

            //解析版快地址
            for (int i = 0; i < categories.size() - 1; i++) {
                String categoriesUrl = "https://kuaihe-720.1919.cn/product/categoryProductId.do?addressLat=31.258492&addressLon=121.595534&cityId=289&depotId=358847913377927168";
                JSONObject jsonObject = JSONObject.parseObject(categories.get(i));
                String categorieId = jsonObject.getString("id");
                CrawlerRequestRecord goodsListRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .httpUrl(categoriesUrl)
                        .recordKey(categoriesUrl + categorieId)
                        .releaseTime(System.currentTimeMillis())
                        .notFilterRecord()
                        .build();
                HttpRequest httpRequest = goodsListRecord.getHttpRequest();
                httpRequest.setMethod(HttpConstant.Method.POST);
                httpRequest.addHeader("Host", "kuaihe-720.1919.cn");
                httpRequest.addHeader("User-Agent", "okhttp/3.12.1");
                String page = "0";
                try {
                    httpRequest.setRequestBody(HttpRequestBody.custom(("data=" +
                            "{\"depotId\": 358847913377927168," +
                            "\"pageSize\": 20, " +
                            "\"sort\": \"SALES_VOLUME_DESC\"," +
                            "\"page\": " + page + "," +
                            "\"categoryId\": " + categorieId + "," +
                            "\"customFlag\": false}").getBytes("UTF-8"), FORM, "UTF-8"));
                } catch (UnsupportedEncodingException e) {
                    log.error(e.getMessage());
                }
                parsedLinks.add(goodsListRecord);
            }

        } else if (requestUrl.matches(goodsListUrl)) {
            List<String> productIds = new ArrayList<>();
            try {
                productIds = httpPage.getJson().jsonPath($_type + ".data.productIds").all();
            } catch (Exception e) {
                log.error(" productIds page down load is fail ! will retry");
                requestAgainCrawlerRecord(parsedLinks,crawlerRequestRecord);
                return parsedLinks;
            }


            //商品列表翻页
            JSONObject jsonObject = JSONObject.parseObject(new String(httpPage.getRequest().getRequestBody().getBody()).split("=")[1]);
            JSONObject nextJsonObject = new JSONObject();
            Set set = jsonObject.toJavaObject(Map.class).keySet();
            for (Object o : set) {
                String key = String.valueOf(o);
                if ("page".equals(key)) {
                    nextJsonObject.put(key, (jsonObject.getIntValue(key) + 1));
                }else{
                    nextJsonObject.put(key, jsonObject.get(key));
                }
            }
            CrawlerRequestRecord turnRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRequestRecord.getRecordKey() + "&page=" + jsonObject.getString("page"))
                    .releaseTime(System.currentTimeMillis())
                    .build();
            HttpRequest httpRequest = turnRecord.getHttpRequest();
            httpRequest.setMethod(HttpConstant.Method.POST);
            httpRequest.setHeaders(crawlerRequestRecord.getHttpRequest().getHeaders());
            String bodyString = "data=" + JSONObject.toJSONString(nextJsonObject);
            try {
                httpRequest.setRequestBody(HttpRequestBody.custom(bodyString.getBytes("UTF-8"), FORM, "UTF-8"));
            } catch (UnsupportedEncodingException e) {
                log.error(e.getMessage());
            }
            parsedLinks.add(turnRecord);


            //商品详细信息地址解析
            for (String productId : productIds) {
                String productUrl = "https://kuaihe-720.1919.cn/product/detail.do?addressLat=31.258525&addressLon=121.595488&cityId=289&depotId=358847913377927168&provinceId=289";

                CrawlerRequestRecord productRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .httpUrl(productUrl)
                        .recordKey(productUrl + productId)
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(article)
                        .build();
                HttpRequest productRecordHttpRequest = productRecord.getHttpRequest();
                productRecordHttpRequest.setMethod(HttpConstant.Method.POST);
                productRecordHttpRequest.addHeader("Host", "kuaihe-720.1919.cn");
                productRecordHttpRequest.addHeader("User-Agent", "okhttp/3.12.1");
                try {
                    productRecordHttpRequest.setRequestBody(HttpRequestBody.custom(("data={\"depotId\":358847913377927168,\"id\":" + productId + "}").getBytes("UTF-8"), FORM, "UTF-8"));
                } catch (UnsupportedEncodingException e) {
                    log.error(e.getMessage());
                }
                parsedLinks.add(productRecord);
            }
        }else if(requestUrl.matches(goodsDetailUrl)){
            String code = httpPage.getJson().jsonPath($_type + ".code").get();
            if(!"0".equals(code)){
                log.error("rawText : " + httpPage.getRawText());
                crawlerRequestRecord.setNeedWashPage(false);
            }
        }
        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();

        if (crawlerResultTags.hasDataType(article)) {
            crawlerDataList.add(washArticle(crawlerRecord, page));
        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();
        String articleKey = httpPage.getJson().jsonPath($_type + ".data.id").get();
        CrawlerData crawlerData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                .url(itemUrl)
                .releaseTime(crawlerRequestRecord.getReleaseTime())
                .content(httpPage.getJson().jsonPath($_type + ".data").get())
                .build();
        return crawlerData;
    }


    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {

        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(requestAgainTag)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(requestAgainTag).getVal();
            if (count >= 5) {
                log.error("kuaihe search keyword download page the number of retries exceeds the limit" +
                        ",request url {},download detail {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpUrl(crawlerRecord.getHttpRequest().getUrl())
                .recordKey(crawlerRecord.getRecordKey() + count)
                .releaseTime(crawlerRecord.getReleaseTime())
                .httpHeads(crawlerRecord.getHttpRequest().getHeaders())
                .notFilterRecord()
                .build();
        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
        httpRequest.setMethod(HttpConstant.Method.POST);
        httpRequest.setRequestBody(crawlerRequestRecord.getHttpRequest().getRequestBody());

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (requestUrl.matches(goodsDetailUrl)) {
            crawlerRequestRecord.setNeedWashPage(true);
            crawlerRequestRecord.tagsCreator().resultTags().addResultDataType(article);
        }

        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(requestAgainTag, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

}
